package crawler.web.component.upengyou;

import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import library.Tracer;
import library.bean.Coordinator;
import library.functions.Function1PR;
import library.functions.Function2PR;
import library.wapper.BoolWapper;

import org.apache.commons.httpclient.HttpException;
import org.apache.commons.lang.StringUtils;
import org.htmlparser.Node;
import org.htmlparser.Parser;
import org.htmlparser.nodes.TextNode;
import org.htmlparser.tags.Bullet;
import org.htmlparser.tags.CompositeTag;
import org.htmlparser.tags.Div;
import org.htmlparser.tags.Html;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.tags.MetaTag;
import org.htmlparser.tags.ParagraphTag;
import org.htmlparser.tags.Span;
import org.htmlparser.util.ParserException;

import crawler.web.beans.CrawlPageContext;
import crawler.web.beans.CrawlURL;
import crawler.web.component.CommonHtmlProcessor;
import crawler.web.component.HtmlDigger;
import crawler.web.component.IComponent;
import crawler.web.engine.config.Suite;
import crawler.web.utils.MyHttpClient;

/**
 * @author waynechen Mar 6, 2012 12:03:02 PM
 */
public class HtmlProcessorUPENGYOU extends CommonHtmlProcessor implements IComponent {

	//--------------------附加信息抓取--------------------------------
	private static final Pattern mapPattern = Pattern.compile("showMap\\('(.*)',(.*)\\)");
	private static final String mapURL = "http://www.upengyou.com/index.php?r=Map&type=%s&id=%s";

	@Override
	public boolean additionRequest(final MyHttpClient client, CrawlPageContext context) {
		//请求下面地址, 获取经纬度
		//http://www.upengyou.com/index.php?r=Map&type=AREA&id=2903
		//从url中获取出<a onclick="showMap('DEST',100001)" href="#!">查看地图</a>
		final Parser parser = new Parser();
		try {
			parser.setInputHTML(context.getUrl().getHtml());
			parser.setEncoding(context.getUrl().getEncoding());
			Html html = HtmlDigger.findHtmlNode(parser);
			if (html != null) {
				final BoolWapper notCatch = new BoolWapper(true);
				HtmlDigger.travelTree(html, context, true, new Function2PR<Node, Object, Boolean>() {
					@Override
					public Boolean apply(Node n, Object b) {
						CrawlPageContext context = (CrawlPageContext) b;
						LinkTag a = (LinkTag) n;
						String attribute = a.getAttribute("onclick");
						Matcher m = mapPattern.matcher(attribute);
						//找出type与id
						if (m.find()) {
							String type = m.group(1);
							String id = m.group(2);
							CrawlURL url = new CrawlURL(String.format(mapURL, type, id));

							try {
								//暂停5秒后抓取
								Thread.sleep(5000);
								//获取html
								String htmlStr = client.getHtml(url);
								if (StringUtils.isEmpty(htmlStr)) {
									notCatch.bool = false;
									return null;
								}
								parser.setInputHTML(htmlStr);
								parser.setEncoding(url.getEncoding());
								Html html = HtmlDigger.findHtmlNode(parser);
								if (html != null) {
									//从中获取经纬度
									Coordinator coor = HtmlDigger.GoogleMap.parseCoorFromHtml(html);
									if (StringUtils.isNotBlank(coor.getLat()) && StringUtils.isNotBlank(coor.getLng())) {
										context.getLocation().setLat(coor.getLatDouble());
										context.getLocation().setLng(coor.getLngDouble());
									}
								}
								return true;
							} catch (HttpException e) {
								Tracer.info(e.getMessage());
								notCatch.bool = false;
							} catch (IOException e) {
								Tracer.info(e.getMessage());
								notCatch.bool = false;
							} catch (Exception e) {
								Tracer.info("找不到经纬度: " + context.getUrl().getOriUrl());
								Tracer.exception(e);
								notCatch.bool = false;
							}
						}
						return null;
					}
				}, new Function1PR<Node, Boolean>() {
					@Override
					public Boolean apply(Node a) {
						if (a instanceof LinkTag) {
							LinkTag l = (LinkTag) a;
							if (l.getLinkText().contains("查看地图")) {
								String attribute = l.getAttribute("onclick").toLowerCase();
								if (attribute.contains("showmap"))
									return true;
							}
						}
						return false;
					}
				});
				return notCatch.bool;
			}
		} catch (ParserException e) {
			Tracer.exception(e);
			return false;
		}
		return true;
	}

	//--------------------页面分析--------------------------------
	@Override
	protected boolean dig(Html root, CrawlPageContext context) throws Exception {

		//只分析 meta
		HtmlDigger.digSubNodes(MetaTag.class, root, context, new Function2PR<Node, Object, Node>() {
			@Override
			public Node apply(Node a, Object context) {
				HtmlDigger.parseMetaTag((CrawlPageContext) context, (MetaTag) a);
				return null;
			}
		});

		//只分析 div
		HtmlDigger.digSubNodes(Div.class, root, context, new Function2PR<Node, Object, Node>() {
			@Override
			public Node apply(Node a, Object b) {
				final Div div = (Div) a;
				CrawlPageContext context = (CrawlPageContext) b;

				//导航信息
				if ("location".equals(div.getAttribute("class"))) {
					HtmlDigger.digAllText(div, context.getNavigate(), null, null);
				}
				//处理 <div class="w425 fl">
				else if ("w425 fl".equals(div.getAttribute("class"))) {
					context.increasePerfectionScore();
					if (!dig_w425_fl(div, context))
						return null;
				}
				//处理 <div id="area_traffic">
				else if ("area_traffic".equals(div.getAttribute("id"))) {
					if (!dig_area_traffic(div, context, context.getLocation().getTraffic()))
						return null;
				}
				//处理 <div class="route bor-rad">
				else if ("route bor-rad".equals(div.getAttribute("class"))) {
					if (!dig_area_traffic(div, context, context.getLocation().getTraffic()))
						return null;
				}
				//处理 <div class="main-con">
				else if ("main-con".equals(div.getAttribute("class"))) {
					if (!dig_area_traffic(div, context, context.getSight().getTextBuffer()))
						return null;
				}
				//其它情况
				else {
					return null;
				}

				//如果这个节点被处理过则返回其
				//返回的节点和其子节点都会被删除
				return div;
			}
		});

		//对于已知的网站, 哪些是垃圾数据很容易判断
		processPageType(context);

		//从导航信息中取出地址信息
		processLocation(context);

		//建立identity
		context.getLocation().autoFillIdentity();

		//如果这个是路线网页则删除sight.textBuffer
		if (context.getUrl().getOriUrl().contains("traffic")) {
			context.getSight().getTextBuffer().setLength(0);
		}

		return true;
	}

	public void processLocation(CrawlPageContext context) {
		String[] loc = context.getNavigate().toString().replaceAll("\\s", "").split(">");
		//[当前位置：目的地, 中国, 广东, 广州, 碧水湾温泉度假村]
		if (loc.length > 1) {
			context.getLocation().setCountry(loc[1]);
		}
		if (loc.length > 2) {
			context.getLocation().setProvince(loc[2]);
		}
		if (loc.length > 3) {
			context.getLocation().setCity(loc[3]);
		}
		if (loc.length > 4) {
			if (loc[4].endsWith("区"))
				context.getLocation().setBusinessArea$sightArea(loc[4]);
			else
				context.getLocation().setStreet$sight(loc[4]);
		}
		if (loc.length > 5) {
			context.getLocation().setBusinessArea$sightArea(loc[4]);
			context.getLocation().setStreet$sight(loc[5]);
		}
	}

	public void processPageType(CrawlPageContext context) {
		//没有有用信息, 被认为是垃圾网页
		if (context.getPerfectionScore() <= 0) {
			context.setPageType(Suite.PageType.GARBAGE.getExpr());
		}
		//最后设置为景点数据
		else {
			context.setPageType(Suite.PageType.SIGHT.getExpr());
		}
	}

	private Boolean dig_area_traffic(final Div root, final CrawlPageContext context, StringBuilder sb) {
		return HtmlDigger.digSubNodes(Bullet.class, root, sb, new Function2PR<Node, Object, Node>() {
			@Override
			public Node apply(Node a, Object b) {
				Bullet li = (Bullet) a;
				if ("line-h22 mt25".equals(li.getAttribute("class"))) {
					context.increasePerfectionScore();
					HtmlDigger.digAllText(li, (StringBuilder) b, "", null);
				}
				return null;
			}
		});
	}

	private Boolean dig_w425_fl(final Div root, CrawlPageContext context) {
		return HtmlDigger.digSubNodes(CompositeTag.class, root, context, new Function2PR<Node, Object, Node>() {
			@Override
			public Node apply(Node a, Object b) {
				CompositeTag subNode = (CompositeTag) a;
				CrawlPageContext context = (CrawlPageContext) b;
				String cls = subNode.getAttribute("class");
				if (cls == null)
					cls = "";
				//正文信息
				if (subNode instanceof Div && cls.contains("more-box") && cls.contains("line-h24")) {
					HtmlDigger.digAllText(subNode, context.getSight().getTextBuffer(), "", new Function1PR<Node, Boolean>() {
						@Override
						public Boolean apply(Node a) {
							TextNode text = (TextNode) a;
							if (text.getText().equals("[收起]"))
								return false;
							return true;
						}
					});
				}
				//门票价格
				else if (subNode instanceof Div && "book-tic".equals(cls)) {
					StringBuilder sb = new StringBuilder();
					HtmlDigger.digAllText(subNode, sb, "", new Function1PR<Node, Boolean>() {
						@Override
						public Boolean apply(Node a) {
							if (a.getText().startsWith("门票价格"))
								return true;
							else
								return false;
						}
					});
					context.getSight().setPrice(sb.toString().replaceAll("门票价格|：|:", ""));
				}
				//正文属性
				else if (subNode instanceof ParagraphTag && "line-h24".equals(cls)) {
					if (subNode.getParent().equals(root)) {
						String l = subNode.getStringText().trim();
						if (l.startsWith("景区等级:")) {
							l = l.replace("景区等级:", "").trim();
							context.getSight().setSightRank(l);
						} else if (l.startsWith("电话:")) {
							l = l.replace("电话:", "").trim();
							context.getSight().setPhone(l);
						} else if (l.startsWith("开放时间:")) {
							l = l.replace("开放时间:", "").trim();
							context.getSight().setBusinessHours(l);
						} else if (l.startsWith("地址:")) {
							l = l.replace("地址:", "").trim();
							context.getLocation().setRawLocation(l);
						} else if (l.startsWith("门票价格")) {
							l = l.replaceAll("门票价格|：|:", "").trim();
							context.getSight().setPrice(l);
						}
					}
				}
				//获取正文评分
				else if (subNode instanceof Div && StringUtils.isEmpty(cls)) {
					Boolean getInfo = HtmlDigger.travelTree(subNode, context.getSight().getScore(), false, new Function2PR<Node, Object, Boolean>() {
						@Override
						public Boolean apply(Node a, Object b) {
							if (a instanceof Span) {
								Span span = (Span) a;
								StringBuilder sb = (StringBuilder) b;
								String attr = (span.getAttribute("class") + "").trim();
								if ("col-org ml10".equals(attr)) {
									HtmlDigger.digAllText(span, sb, null, null);
									return true;
								}
							}
							return false;
						}
					}, null);
					//没有获取到信息则不删除此节点
					if (!getInfo) {
						return null;
					}
				}
				//其它情况
				else {
					//不处理, 也不删除此节点
					return null;
				}
				return subNode;
			}
		});
	}

	//-----------------------------------url选择------------------------------------------------------

	private static final Pattern pattern = Pattern.compile("^http://www\\.upengyou\\.com.*");

	@Override
	public boolean accept(CrawlURL url) {
		return accept(url, pattern);
	}

}

//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//